# -*- coding: utf-8 -*- 
from requests import get
from pyquery import PyQuery as pq
import xlrd
from xlutils.copy import copy
import time

headers = {
    'Host': 'www.findamasters.com',
    'Connection': 'keep-alive',
    'Cache-Control': 'max-age=0',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.104 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'zh-CN,zh;q=0.8,en;q=0.6',
}
# 37,保存地区方便后期检索
link1 = "https://www.educations.com/search/game-design-development/c2693"
# 308
link2 = "https://www.findamasters.com/masters-degrees/?Keywords=game&PP=30&PG="
# 57
link3 = "https://www.hotcoursesabroad.com/study/searchajax.html?search=coursesearch&catCode=CB.3211-4&subQualId=57&parentQualId=3&nationCode=39&nationCntryCode=39&studyAbroad=N&studyOnline=N&studyCross=N&studyDomestic=N&studyPartTime=N&pageNo="
# save_file1 = open("schools/file1.txt", "a", encoding='utf-8')
# save_file2 = open("schools/file2.txt", "a", encoding='utf-8')
# save_file3 = open("schools/file3.txt", "a", encoding='utf-8')

read_book = xlrd.open_workbook("schools.xls")
write_book = copy(read_book)

def grab_link1():
	sheet = write_book.get_sheet(0)
	count = 5
	page = pq(get(link1, headers = headers).text)
	list_col = page('.ext-serp-rows').items()
	# 13:14
	for col in list(list_col)[5:37]:
		col_degree = col('a.ext-serp-edu').text()
		col_link = col('a.ext-serp-edu').attr.href
		col_school = col('span.ext-serp-ins').text()
		col_level = col('div.ext-serp-type span').text()
		col_place = col('div.ext-serp-place span').text()
		sheet.write(count+1, 0, col_degree)
		sheet.write(count+1, 1, col_school)
		sheet.write(count+1, 2, col_level)
		sheet.write(count+1, 3, col_place)
		col_page = pq("https://www.educations.com" + col_link)
		span_labels = list(col_page('div.education-summary-box span.education-summary-label').items())
		span_values = list(col_page('div.education-summary-box span.education-summary-value').items())
		span_dict = dict(zip([label.text() for label in span_labels],
		 	[value.text() for value in span_values]))
		sheet.write(count+1, 4, ",".join(str(t) for t in span_dict.items()))
		col_desc = col_page('div.education-description-wrapper').text()
		col_img = col_page('div.education-description-wrapper img').attr.src
		sheet.write(count+1, 5, col_img)
		sheet.write(count+1, 6, col_desc)
		col_qual = col_page('div#lcb-EducationQualification section>*:not(div)').text()
		col_Ieltsqual = col_page('div#lcb-IeltsEducationQualification section>*:not(div)').text()
		sheet.write(count+1, 7, col_qual)
		sheet.write(count+1, 8, col_Ieltsqual)
		col_diploma = col_page('div.lcb-EducationDegree section').text()
		col_tuition = col_page('div.lcb-EducationCosts section p').text()
		col_address = col_page('div.adr').text()
		col_tel = col_page('span#Telephone').text()
		col_url = col_page('span.txtSmall>a.url').text()
		sheet.write(count+1, 9, col_diploma)
		sheet.write(count+1, 10, col_tuition)
		sheet.write(count+1, 11, col_address)
		sheet.write(count+1, 12, col_tel)
		sheet.write(count+1, 13, col_url)
		count+=1
		write_book.save('schools.xls')
		print(str(count))


def grab_link2():
	sheet = write_book.get_sheet(1)
	count = 274
	# 30个每页，&PG=2，共11页
	for i in range(10, 12):
		# list_in_page = pq(get(link2 + str(i), headers = headers).text)('div.courses > div.row.tight').items()
		list_in_page = pq(get(link2 + str(i), headers = headers).text)('div.courses > div.resultsRow.basic').items()
		# print((list(list_in_page))[0])
		# for col in list(list_in_page)[0: 8]:
		for col in list(list_in_page):
			col_degree = col('a.courseLink').text()
			col_link = col('a.courseLink').attr.href
			col_inst = col('a.instLink').text()
			col_dept = col('a.deptLink').text()
			sheet.write(count+1, 0, col_degree)
			sheet.write(count+1, 1, col_inst)
			sheet.write(count+1, 2, col_dept)
			try:
				col_page = pq(get('https://www.findamasters.com' + col_link, headers=headers).text)
			except Exception as e:
				print("network error")
				return
			else:
				col_country = col_page('h2.ctryLink>a').text()
				col_city = col_page('h2.cityLink').text()
				col_desc = col_page('div.course-description div.description-content').text()
				# print(col_country)
				# print(col_city)
				
				sheet.write(count+1, 3, col_country)
				sheet.write(count+1, 4, col_city)
				sheet.write(count+1, 5, col_desc)
				info_keys = list(col_page('div.extra-course-content div.panel li>h4').items())
				info_values = list(col_page('div.extra-course-content div.panel li').items())
				info_dict = dict(zip([key.text() for key in info_keys],
					[value('p').text() for value in info_values]))
				sheet.write(count+1, 6, ','.join(str(t) for t in info_dict.items()))
				# save_file2.write('\n')
				# col_requ = col_page('div.entry-requirements-area').text()
				# video_links = ", ".join(i.attr.href for i in list(col_page('ul#VideoList li link').items()))s
				# print()
				count+=1
				write_book.save('schools.xls')
				print(str(count))
				time.sleep(6)


def grab_link3():
	# 57,5页
	# uni_sheet = write_book.get_sheet(2)
	course_sheet = write_book.get_sheet(3)
	uni_count = 0
	course_count = 0
	for i in range(1, 6):
		list_in_page = list(pq(link3 + str(i))('div.nrmtab').items())#12
		# print(len(list_in_page))
		for col in list_in_page:
			# col_university = col('div.sr_nam h2').text()
			# col_uni_link = col('div.sr_nam h2 a').attr.href
			# col_country = col('div.sr_nam span.grey').text()
			# col_rank = col('p.flft').text()
			# save_file3.write
			# uni_sheet.write(uni_count + 1, 0, col_university)
			# uni_sheet.write(uni_count + 1, 1, col_country)
			# uni_sheet.write(uni_count + 1, 2, col_rank)
			# print(col_university, col_country, col_rank)
			# 学校页面
			# col_uni_page = pq(col_uni_link)
			# col_desc = col_uni_page('div.pl_cde').text()
			# # # col_about = col_uni_page('div.rgt_cnt.oview').text().encode('utf-8')
			# col_english = col_uni_page('div.flud.nw_dpt').text()
			# uni_sheet.write(uni_count + 1, 3, col_desc)
			# uni_sheet.write(uni_count + 1, 4, col_uni_page('div.rgt_cnt.oview').text())
			# uni_sheet.write(uni_count + 1, 5, col_english)
			# rate_keys = list(col_uni_page('div.rat_nw b').items())
			# rate_values = list(col_uni_page('div.hldr.pr_pd p.val_rat').items())
			# rate_dict = dict(zip([key.text() for key in rate_keys], 
			# 	[value.text() for value in rate_values]))
			# uni_sheet.write(uni_count + 1, 6, ','.join(str(t) for t in rate_dict.items()))
			# score_keys = list(col_uni_page('div.hldr.key_pat div.str1 p').items())
			# score_values = list(col_uni_page('div.hldr.key_pat div.str1 div.bar_rat').items())
			# score_dict = dict(zip([key.text() for key in score_keys], 
			# 	[value.text() for value in score_values]))
			# # # print(score_dict)
			# uni_sheet.write(uni_count + 1, 7, ','.join(str(t) for t in score_dict.items()))
			# # # col_cost = col_uni_page('div.key_rgt.p_ct.fw_im').text().encode('utf-8')
			# # # print(col_cost)
			# uni_sheet.write(uni_count + 1, 8, col_uni_page('div.key_rgt.p_ct.fw_im').text())
			# # 该学校的课程页面
			col_course_page = pq("https://www.hotcoursesabroad.com" + col('a.whpd_clk').attr.href)
			course_list = col_course_page('div.pr_rslt.sr.nrmtab').items()
			uni_count += 1
			for course in course_list:
				# course_sheet.write(course_count+1, 0, col_university)
				# course_name = course('div.sr_nam').text()
				# course_sheet.write(course_count+1, 1, course_name)
				# # [1:]学位，时间，开始时间，学费
				# course_info_list = list(course('div.hldr.mb10').items())[1:]
				# col_degree = course_info_list[0]('div.pr_hd_rgt').text()
				# col_time = course_info_list[1]('div.pr_hd_rgt').text()
				# course_sheet.write(course_count+1, 2, col_degree)
				# course_sheet.write(course_count+1, 3, col_time)

				course_link = course('a.whpd_clk').attr.href
				course_page = pq(course_link)
				# course_intro = course_page('section#what-will-i-learn div.oview').text()
				# course_depart = course_page('section#Which-department-am-i-in div.oview').text()
				course_info_list = list(course_page('div.addrs div.tut_fs').items())
				course_sheet.write(course_count+1, 4, course_info_list[1]('div.fs_rgt').text())
				# course_sheet.write(course_count+1, 5, course_info_list[0]('div.rat_nw').text())
				# course_sheet.write(course_count+1, 6, course_info_list[-1]('div.fs_rgt').text())
				write_book.save('schools.xls')
				print(str(course_count), str(uni_count))
				course_count += 1
				time.sleep(7)



if __name__ == '__main__':
	# grab_link1()
	# grab_link2()
	grab_link3()
	# save_file2.close()
